﻿using MongoDB.Bson;
using MongoDB.Driver;
using System;
using System.Collections.Generic;
using System.ComponentModel;
using System.Data;
using System.Drawing;
using System.IO;
using System.Linq;
using System.Net;
using System.Text;
using System.Threading.Tasks;
using System.Windows.Forms;
using System.Xml;

namespace DataBaseTool
{
    public partial class Form4 : Form
    {
        public Form4()
        {
            InitializeComponent();
        }

        Boolean flag = true;

        private void button2_Click(object sender, EventArgs e)
        {
            flag = false;
            MessageBox.Show("stop");
           
        }

        private void button1_Click(object sender, EventArgs e)
        {
            int index=comboBox1.SelectedIndex;

            switch (index)
            {
                case 0:
                    
                    getDataOne(100);
                    break;
                case 1:
                    getDataTwo(100);
                    break;
                case 2:
                    getDataThree(100);
                    break;
                case 3:
                    getDataFour(100);
                    break;
                case 4:
                    getDataFive(100);
                    break;
            }
           

           


        }

        private void getDataOne(int count)
        {
            label1.Text = "searching...";
             
            

            for (int i = 0; i < count; i++)
            {
                label2.Text = i.ToString();
                string rs = HttpPost("https://www.misuland.com/home/refreshPAE.do", "pageIndex=" + i + "&keyWords=&typeMore=0");
                int start_index = rs.IndexOf("result-1");
                int end_index = rs.IndexOf("numList");
                start_index = start_index + 38;
                end_index = end_index - 12;
                string rs1 = rs.Substring(start_index, end_index - start_index);
                rs1 = "<div>" + rs1;
                while (rs1.IndexOf("<h2>") > 0)
                {
                    int start1 = rs1.IndexOf("<h2>");
                    int start2 = rs1.IndexOf("</h2>");

                    string ct = rs1.Substring(start1 + 4, start2 - start1 - 4);


                    int st1 = ct.IndexOf("href");

                    int st2 = ct.IndexOf("target");

                    string href = ct.Substring(st1 + 6, st2 - st1 - 8);

                    href = "https://www.misuland.com" + href;

                    int st3 = ct.IndexOf("title");

                    int st4 = ct.IndexOf(">");

                    string title = ct.Substring(st3 + 7, st4 - st3 - 8);

                    // Console.WriteLine(title);

                    BsonDocument document = new BsonDocument();

                    document.Add("href", href);

                    long ex = collection.Find(document).CountDocuments();
                    if (ex == 0)
                    {


                        int ht_indx = href.LastIndexOf("https://");
                        href = href.Substring(ht_indx, href.Length - ht_indx);
                        string page_content = HttpGet(href,"UTF-8", Encoding.UTF8,"");
                        //Console.WriteLine(page_content);

                        int pg_start1 = page_content.IndexOf("proSentent");

                        int pg_start2 = page_content.IndexOf("proBlock");

                        document.Add("title", title);

                        document.Add("source", "www.misuland.com");

                        if (pg_start1 < 0)
                        {
                            document.Add("need", "");
                            document.Add("id", getAutoIncreament("needs"));
                            collection.InsertOne(document);
                            continue;
                        }

                        int need_start = page_content.IndexOf("everyPro");

                        int need_end = page_content.IndexOf("</textarea");

                        string need_str = page_content.Substring(need_start, need_end - need_start);

                        int n_st = need_str.IndexOf("<textarea");

                        string need = need_str.Substring(n_st, need_str.Length - n_st);

                        int n_start = need.IndexOf(">");

                        need = need.Substring(n_start + 1, need.Length - n_start - 1);


                        document.Add("need", need);
                        document.Add("id", getAutoIncreament("needs"));
                        collection.InsertOne(document);



                    }
                    rs1 = rs1.Substring(start2 + 5);

                }
                if (i < 100)
                {
                    progressBar1.Value = i;
                }
            }

            progressBar1.Value = 100;
            BsonDocument doc = new BsonDocument();
            doc.Add("source", "www.misuland.com");
            long docs_count = collection.Find(doc).CountDocuments();

            label1.Text = " www.misuland.com  found " + docs_count;


        }


        private void getDataTwo(int count)
        {

            label1.Text = "searching...";

            

            for (int i = 0; i < count; i++)
            {
                label2.Text = i.ToString();
                string rs = HttpPost("https://www.sxsoft.com/page/project/search/0/true", "page="+(i+1)+"&category=&keyword=&suozaidi=&yusuan=&xiangmuzhuangtai=jbz&fromTime=&toTime=&suoxujishu=&orderKey=");

                

                int start_index = 0;// rs.IndexOf("projectLists");
                string rs2=rs.Substring(start_index, rs.Length - start_index);

                int start_index2 = rs2.IndexOf("<ul");
                int end_index2=rs2.IndexOf("</ul>");

                if (start_index2 < 0)
                {
                    continue;
                }

                string rs3=rs2.Substring(start_index2, end_index2 - start_index2);
               
                
                while (rs3.IndexOf("<h4>") > 0)
                {
                    int start1 = rs3.IndexOf("<h4>");
                    int start2 = rs3.IndexOf("</h4>");

                    string ct = rs3.Substring(start1 + 4, start2 - start1 - 4);

                    

                    int st1 = ct.IndexOf("href");

                    int st2 = ct.IndexOf("target");

                    string href = ct.Substring(st1 + 6, st2 - st1 - 8);

                    href = "https://www.sxsoft.com" + href;
                    
                    int st3 = ct.IndexOf("</a>");

                    string title_str = ct.Substring(st2, st3 - st2);

                    int t_start1=title_str.IndexOf(">");

                    string title=title_str.Substring(t_start1 + 1, title_str.Length-t_start1- 1);
                    

                  // Console.WriteLine(title+"====="+href);

                   
                  


                    BsonDocument document = new BsonDocument();

                    document.Add("href", href);

                    long ex = collection.Find(document).CountDocuments();
                    if (ex == 0)
                    {


                        int ht_indx = href.LastIndexOf("https://");
                        href = href.Substring(ht_indx, href.Length - ht_indx);
                        string page_content = HttpGet(href,"UTF-8", Encoding.UTF8, "");
                       // Console.WriteLine(page_content);
                        
                         

                        int pg_start1 = page_content.IndexOf("项目内容描述");

                        int pg_start2 = page_content.IndexOf("相关项目推荐");

                        document.Add("title", title);

                        document.Add("source", "www.sxsoft.com");

                        if (pg_start1 < 0)
                        {
                            document.Add("need", "");
                            document.Add("id", getAutoIncreament("needs"));
                            collection.InsertOne(document);
                            continue;
                        }

                        string need_string = page_content.Substring(pg_start1, pg_start2 - pg_start1);

                        int div_index = need_string.IndexOf("<div");

                        need_string = need_string.Substring(div_index, need_string.Length - div_index);


                        int need_start = need_string.IndexOf("<p");

                        int need_end = need_string.IndexOf("</p>");

                        string need = need_string.Substring(need_start+3, need_end - need_start-3);
                        
                        document.Add("need", need);
                        document.Add("id", getAutoIncreament("needs"));
                        collection.InsertOne(document);
                        
                    }
                    rs3 = rs3.Substring(start2 + 5);

                }
                if (i < 100)
                {
                    progressBar1.Value = i;
                }
            }

            progressBar1.Value = 100;
            BsonDocument doc = new BsonDocument();
            doc.Add("source", "www.sxsoft.com");
            long docs_count = collection.Find(doc).CountDocuments();

            label1.Text = " www.sxsoft.com  found "+ docs_count;

        }


        private void getDataThree(int count)
        {

            label1.Text = "searching...";

             

            for (int i = 0; i < count; i++)
            {
                label2.Text = i.ToString();
                string page_addr = "tasklist-page-"+i;
                if (i <2 )
                {
                    page_addr = "tasklist";
                }
                string rs = HttpGet("http://www.a5.cn/"+page_addr+".html","gbk", Encoding.GetEncoding("GBK"), "");

                
               // Console.WriteLine(rs);

               

                int start_index = rs.IndexOf("m-tk-list clearfix");
                string rs2 = rs.Substring(start_index, rs.Length - start_index);

                int start_index2 = rs2.IndexOf("<ul");
                int end_index2 = rs2.IndexOf("</ul>");

                if (start_index2 < 0)
                {
                    continue;
                }

                string rs3 = rs2.Substring(start_index2, end_index2 - start_index2);


                while (rs3.IndexOf("<h3>") > 0)
                {
                    int start1 = rs3.IndexOf("<h3>");
                    int start2 = rs3.IndexOf("</h3>");

                    string ct = rs3.Substring(start1 + 4, start2 - start1 - 4);



                    int st1 = ct.IndexOf("href");

                    int st2 = ct.IndexOf("target");

                    string href = ct.Substring(st1 + 5, st2 - st1 - 6);

                    href = "http://www.a5.cn/" + href;

                    int st3 = ct.IndexOf("</a>");

                    string title_str = ct.Substring(st2, st3 - st2);

                    int t_start1 = title_str.IndexOf("i>");

                    string title = title_str.Substring(t_start1 + 2, title_str.Length - t_start1 - 2);
                    
                    
                    BsonDocument document = new BsonDocument();

                    document.Add("href", href);

                    long ex = collection.Find(document).CountDocuments();
                    if (ex == 0)
                    {


                        int ht_indx = href.LastIndexOf("http://");
                        href = href.Substring(ht_indx, href.Length - ht_indx);
                        string page_content = HttpGet(href,"gbk", Encoding.GetEncoding("GBK"), "");
                       //  Console.WriteLine(page_content);



                        int pg_start1 = page_content.IndexOf("任务描述");

                        int pg_start2 = page_content.IndexOf("提交我的报价");

                        document.Add("title", title);

                        document.Add("source", "www.a5.cn");

                        if (pg_start1 < 0)
                        {
                            document.Add("need", "");
                            document.Add("id", getAutoIncreament("needs"));
                            collection.InsertOne(document);
                            continue;
                        }

                        string need_string = page_content.Substring(pg_start1, pg_start2 - pg_start1);

                        int div_index = need_string.IndexOf("m-task-arc clearfix");

                        need_string = need_string.Substring(div_index, need_string.Length - div_index);


                        int need_start = need_string.IndexOf(">");

                        int need_end = need_string.IndexOf("</div>");

                        string need = need_string.Substring(need_start + 1, need_end - need_start - 1);

                       // Console.WriteLine(need);

                        document.Add("need", need);
                        document.Add("id", getAutoIncreament("needs"));
                        collection.InsertOne(document);

                    }
                    rs3 = rs3.Substring(start2 + 5);

                }
                if (i < 100)
                {
                    progressBar1.Value = i;
                }
            }

            progressBar1.Value = 100;
            BsonDocument doc = new BsonDocument();
            doc.Add("source", "www.a5.cn");
            long docs_count = collection.Find(doc).CountDocuments();

            label1.Text = " www.sxsoft.com  found " + docs_count;

        }


        private void getDataFour(int count)
        {

            label1.Text = "searching...";

             

            for (int i = 0; i < count; i++)
            {
                string page_addr = "task/taskMarket!taskFilter?searchKeyword=&page=" + i;
                if (i < 2)
                {
                    page_addr = "task/taskFilter";
                }
                string rs = HttpGet("http://www.junengli.com/" + page_addr, "UTF-8", Encoding.UTF8, "");

                if (string.IsNullOrEmpty(rs))
                {
                    continue;
                }
                int start_index = rs.IndexOf("loadDiv");
                string rs2 = rs.Substring(start_index, rs.Length - start_index);

                int start_index2 = rs2.IndexOf("<dl");
                int end_index2 = rs2.IndexOf("</dl>");

                if (start_index2 < 0)
                {
                    continue;
                }

                string rs3 = rs2.Substring(start_index2, end_index2 - start_index2);


                while (rs3.IndexOf("<dd>") > 0)
                {
                    int start1 = rs3.IndexOf("<dd>");
                    int start2 = rs3.IndexOf("</dd>");

                    string ct = rs3.Substring(start1 + 4, start2 - start1 - 4);



                    int st1 = ct.IndexOf("href");

                    int st2 = ct.IndexOf("target");

                    string href = ct.Substring(st1 + 6, st2 - st1 - 8);

                    href = "http://www.junengli.com" + href;

                    int st3 = ct.IndexOf("</a>");

                    string title_str = ct.Substring(st2, st3 - st2);

                    int t_start1 = title_str.IndexOf(">");

                    string title = title_str.Substring(t_start1 + 1, title_str.Length - t_start1 - 1);


                    // Console.WriteLine(title+"====="+href);





                    BsonDocument document = new BsonDocument();

                    document.Add("href", href);

                    long ex = collection.Find(document).CountDocuments();
                    if (ex == 0)
                    {


                        int ht_indx = href.LastIndexOf("http://");
                        href = href.Substring(ht_indx, href.Length - ht_indx);
                        string page_content = HttpGet(href, "UTF-8", Encoding.UTF8, "");
                        // Console.WriteLine(page_content);



                        int pg_start1 = page_content.IndexOf("xq_miaoshu xx-miaoshu");

                        int pg_start2 = page_content.IndexOf("任务附件");

                        document.Add("title", title);

                        document.Add("source", "www.junengli.com");

                        if (pg_start1 < 0)
                        {
                            document.Add("need", "");
                            document.Add("id", getAutoIncreament("needs"));
                            collection.InsertOne(document);
                            continue;
                        }

                        string renwu_str = page_content.Substring(pg_start1, pg_start2 - pg_start1);

                        int n_st1=renwu_str.IndexOf(">");
                        int n_st2=renwu_str.IndexOf("</div");

                        string need = "";
                        if (n_st2 > 0)
                        {
                            need = renwu_str.Substring(n_st1 + 1, n_st2 - n_st1 - 1);
                        }
                        
                        document.Add("need", need);
                        document.Add("id", getAutoIncreament("needs"));
                        collection.InsertOne(document);

                    }
                    rs3 = rs3.Substring(start2 + 5);

                }
                if (i < 100)
                {
                    progressBar1.Value = i;
                }
            }

            progressBar1.Value = 100;
            BsonDocument doc = new BsonDocument();
            doc.Add("source", "www.junengli.com");
            long docs_count = collection.Find(doc).CountDocuments();

            label1.Text = " www.junengli.com found " + docs_count;

        }



        MongoClient client = null;

        IMongoDatabase database = null;

        IMongoCollection<BsonDocument> collection = null;

        private void getDataFive(int count)
        {

            label1.Text = "searching...";
             
            for (int i = 0; i < count; i++)
            {
                string page_addr = "task/soft/page"+i+".html";
                if (i < 2)
                {
                    page_addr = "task/soft/";
                }
                string rs = HttpGet("http://www.epwk.com/" + page_addr, "UTF-8", Encoding.UTF8, "");

                if (string.IsNullOrEmpty(rs))
                {
                    continue;
                }
                int start_index = rs.IndexOf("task_class_list_li");
                string rs2 = rs.Substring(start_index, rs.Length - start_index);

                int start_index2 = rs2.IndexOf("<h3");
                int end_index2 = rs2.IndexOf("</h3>");

                if (start_index2 < 0)
                {
                    continue;
                }

                string rs3 = rs2.Substring(start_index2, end_index2 - start_index2);

                rs3 = rs2;


                while (rs3.IndexOf("task_class_list_li_box") > 0)
                {
                    int start1 = rs3.IndexOf("task_class_list_li_box");

                    string rs3_tmp = rs3.Substring(start1+1, rs3.Length - start1-1);

                    int start2 = rs3_tmp.IndexOf("task_class_list_li_box");

                    if (start2 < 0)
                    {
                        continue;
                    }

                    string ct = rs3.Substring(start1 + 4,start2-1);



                    int st1 = ct.IndexOf("href");

                    int st2 = ct.IndexOf("title");
                     
                    string href = ct.Substring(st1 + 6, st2 - st1 - 8);

                   // href = "http://www.epwk.com" + href;

                    int st3 = ct.IndexOf("</a>");

                    string title_str = ct.Substring(st2, st3 - st2);

                    int t_start1 = title_str.IndexOf(">");

                    string title = title_str.Substring(t_start1 + 1, title_str.Length - t_start1 - 1);


                    // Console.WriteLine(title+"====="+href);





                    BsonDocument document = new BsonDocument();

                    document.Add("href", href);

                    long ex = collection.Find(document).CountDocuments();
                    if (ex == 0)
                    {


                        int ht_indx = href.LastIndexOf("http://");
                        href = href.Substring(ht_indx, href.Length - ht_indx);
                        string page_content = HttpGet(href, "UTF-8", Encoding.UTF8, "");
                        // Console.WriteLine(page_content);



                        int pg_start1 = page_content.IndexOf("task-info-content");

                        int pg_start2 = page_content.IndexOf("task-user-info-action");

                        document.Add("title", title);

                        document.Add("source", "www.epwk.com");

                        if (pg_start1 < 0)
                        {
                            document.Add("need", "");
                            document.Add("id", getAutoIncreament("needs"));
                            collection.InsertOne(document);
                            continue;
                        }

                        string renwu_str = page_content.Substring(pg_start1, pg_start2 - pg_start1);

                        int n_st1 = renwu_str.IndexOf(">");
                        int n_st2 = renwu_str.IndexOf("</div");

                        string need = "";
                        if (n_st2 > 0)
                        {
                            need = renwu_str.Substring(n_st1 + 1, n_st2 - n_st1 - 1);
                        }

                        document.Add("need", need);

                        document.Add("id", getAutoIncreament("needs"));

                        collection.InsertOne(document);

                    }
                    rs3 = rs3.Substring(start2 + 20);

                }
                if (i < 100)
                {
                    progressBar1.Value = i;
                }
            }

            progressBar1.Value = 100;
            BsonDocument doc = new BsonDocument();
            doc.Add("source", "www.epwk.com");
            long docs_count = collection.Find(doc).CountDocuments();

            label1.Text = " www.epwk.com found " + docs_count;

        }


        private int getAutoIncreament(string collectionName)
        {
            var  myCollection = database.GetCollection<BsonDocument>("sequence");
            BsonDocument doc = new BsonDocument();
            doc.Add("table_name", collectionName);
            
            long ct=myCollection.Find(doc).CountDocuments();
            int cc = 0;
            if (ct == 0)
            {
                cc = 1;
                doc.Add("count", 1);
                myCollection.InsertOne(doc);
            }
            else
            {

                BsonDocument myDoc = myCollection.Find(doc).First();
                string new_count=myDoc.GetValue("count").ToString();
                try
                {
                    cc = int.Parse(new_count);
                    cc = cc + 1;
                    doc.Add("count", cc);
                    BsonObjectId objectId = new BsonObjectId(myDoc.GetValue("_id").ToString());
                    doc.Add("_id", objectId);
                    myCollection.FindOneAndUpdate(myDoc, doc);
                    
                }
                catch
                {

                }
            }

            return cc;

        }

        int i = 0;

        private void timer1_Tick(object sender, EventArgs e)
        {
           
            
            if(flag)
            {
                i++;
                Console.WriteLine(i);
            }
        }


        public static string HttpGet(string Url,string charset, Encoding encoding,string postDataStr)
        {
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url + (postDataStr == "" ? "" : "?") + postDataStr);
            request.Method = "GET";
            request.ContentType = "text/html;charset="+charset;//GBK
            string retString = "";
            try
            {
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                Stream myResponseStream = response.GetResponseStream();
                StreamReader myStreamReader = new StreamReader(myResponseStream, encoding);// Encoding.UTF8);
                retString = myStreamReader.ReadToEnd();
                myStreamReader.Close();
                myResponseStream.Close();
            }
            catch
            {

            }
            return retString;
        }


        /// <summary> 
        /// POST请求与获取结果 
        /// </summary> 
        public static string HttpPost(string Url, string postDataStr)
        {
            HttpWebRequest request = (HttpWebRequest)WebRequest.Create(Url);
            request.Method = "POST";
            request.ContentType = "application/x-www-form-urlencoded";
            request.ContentLength = postDataStr.Length;
            string retString = "";
            try
            {
                StreamWriter writer = new StreamWriter(request.GetRequestStream(), Encoding.ASCII);
                writer.Write(postDataStr);
                writer.Flush();
                HttpWebResponse response = (HttpWebResponse)request.GetResponse();
                string encoding = response.ContentEncoding;
                if (encoding == null || encoding.Length < 1)
                {
                    encoding = "UTF-8"; //默认编码 
                }
           
            StreamReader reader = new StreamReader(response.GetResponseStream(), Encoding.GetEncoding(encoding));
             retString = reader.ReadToEnd();
            }
            catch
            {

            }
            return retString;
        }

        private void Form4_Load(object sender, EventArgs e)
        {

            string host = "localhost";
            int port = 27021;
            string data_base = "bigdata";
            string user_name = "bigdata";
            string pass_word = "123456";
            string connectionString = "mongodb://" + user_name + ":" + pass_word + "@" + host + ":" + port + "/" + data_base;

            client = new MongoClient(connectionString);

            database = client.GetDatabase(data_base);

            collection = database.GetCollection<BsonDocument>("needs");
            



        }
    }
}
